package com.shujia.spark.mllib

import org.apache.spark.ml.classification.{LogisticRegression, LogisticRegressionModel}
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object Demo6ImageModel {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession
      .builder()
      .master("local[16]")
      .appName("point")
      .getOrCreate()
    import spark.implicits._
    import org.apache.spark.sql.functions._

    /**
     * 1、读取svm格式的数据
     */
    val dataDF: DataFrame = spark
      .read
      .format("libsvm")
      .option("numFeatures", 784) //指定向量的大小
      .load("data/image_data")


    /**
     * 2、将数据拆分成训练集和测试集
     */
    val Array(train: DataFrame, test: DataFrame) = dataDF.randomSplit(Array(0.8, 0.2))

    /**
     * 3、选择算法
     */

    val logisticRegression = new LogisticRegression()


    /**
     * 4、将训练集带入算法训练模型
     */
    val model: LogisticRegressionModel = logisticRegression.fit(train)

    /**
     * 5、将测试集带入模型测试模型准确率
     */

    val testDF: DataFrame = model.transform(test)

    val p: Double = testDF.where($"label" === $"prediction").count().toDouble / testDF.count()

    println(p)

    /**
     * 6、保存模型
     */

    model.write
      .overwrite()
      .save("data/image_model")

    while (true){

    }
  }

}
